
package com.kinth.work.tyc;
import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.*;


import org.apache.log4j.xml.DOMConfigurator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;



import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.PriorityScheduler;
import us.codecraft.webmagic.selector.Html;

/**
 * 
 * <p>
 * title: 抓取慧聪网，4月28日更新。解决之前使用连接池的问题。觉得不用链接池。
 * 1. 因为总的类别有832个。所以不能存到内存里面，只能存到数据库里面。所以使用的方法就是先吧总的数据从到huicong_url1这个
 *    表里面。然后后面每次取一个。因为这个表已经存在，就在下面的程序里面吧这部分代码注释掉了。如果需要，可以去掉注释。
 * 2. 目前采用传入一个类别的网址，然后按照优先级迭代。
 * 3. 可是貌似速度太快会被屏蔽。这里还需要在看看。但是貌似是会反扒屏蔽ip的。之前没有遇到过。
 * 4. 当然，反扒只限于具体网页。
 * </p>
 * <p>
 * 
 * </p>
 * 
 * @author kaikai
 * @2017年4月1日
 */
public class TianYanCha implements PageProcessor
{
	private Site site = Site.me().setUserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 "
			+ "(KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36").setRetryTimes(3).setSleepTime(100);

	public static final String BIG_URL_ENTRY = "http://info.b2b.hc360.com/list/company.shtml";
	public static final String BIG_URL = "http://s.hc360.com/\\?w=[\\s\\S]*?&mychannel=enterprise";
    public static final String BIG_URL2 =  "http://s.hc360.com/\\?w=[\\s\\S]*?&mychannel=enterprise&ee=[0-9]+";
	public  static final String COMPANY_URL = "http://[a-zA-Z0-9]+.b2b.hc360.com/";
	
	public static final int NUM_ONCE = 1;//每次从huicong1里面取出的url数目
	//添加每个链接的延迟，为了不被ip屏蔽
    public static final int SLEEP_TIME = 5000;
    
    //定位文件的地址
   
    
    protected Logger logger = LoggerFactory.getLogger(getClass());
    
	@Override
	public void process(Page page)
	{
		try
		{
			Thread.sleep(1000);
		} catch (InterruptedException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		String html = null;
		try
		{
			html = JSUtil.getParseredHtml2(page.getUrl().toString());
			//html = page.getHtml().toString();
			//System.out.println("name: "+html);
		} catch (Exception e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		System.out.println("测试: 3");
		//Page page2 = new Page();
		page.setRawText(html);
		if(page.getHtml() == null)
		{
			System.out.println("null");
		}
		page.setRawText(html);
		System.out.println("得到最终的html5: "+page.getHtml().toString());
		//System.out.println("得到最终的html0: "+page.getRawText());
		//System.out.println("html:in "+page.getHtml().toString());
//		page.setHtml(html);
//		String name =  page2.getHtml()
//				.xpath("html/body"
//						+ "//div[@id='ng-view']"
//		+ "/div[@class='ng-scope']"
//						+ "//div[@class='container company_container']")
////						+ "//div[@class='company_header_width']/div/span"
//				.toString();
//System.out.println("得到最终的html: "+page.getHtml());

		
	}

	@Override
	public Site getSite()
	{
		return site;
	}
    
	// 一个代替main函数的函数
	public static void runSpider() throws Exception
	{
		int rank = 1;
		
		
		
		System.out.println("爬取类别链接开始");
		TianYanCha tyc = new TianYanCha();
		Spider spider = Spider.create(tyc);
		
		
		spider
		.setScheduler(new PriorityScheduler())
		.addPipeline(new DataPipeline())
		;
		
		while (true)
		{
			ArrayList<TYCurl> urlList = dbUtil.queryUrl("tyc_location", rank, NUM_ONCE);
			TYCurl tycurl = null;
			String url = null;
			String location = null;

			if(urlList.size() == 1)
			{
				tycurl = urlList.get(0);
				url = tycurl.getUrl();
				location = tycurl.getLocation();
				//组合js的url
				
				Request request = new Request(url).putExtra("location", location).setPriority(1);
				
				spider.addRequest(request).run();
				
				while(spider.getStatus()!=Spider.Status.Stopped)
				{
					Thread.sleep(10 * 1000);
				}
				rank ++;
			}
			
		}
		
		
		
	
}

	public static void main(String[] args) throws IOException
	{
//		DOMConfigurator.configure(System.getProperty("user.dir")+File.separator+"log4j.xml");
//		try
//		{
//			runSpider();
//		} catch (Exception e)
//		{
//			// TODO Auto-generated catch block
//			e.printStackTrace();
//		}

		TianYanCha huiCong = new TianYanCha();
		Spider spider = Spider.create(huiCong);
		
		
		spider
		.setScheduler(new PriorityScheduler())
		//.addPipeline(new DataPipeline())
		//http://www.tianyancha.com/company/2338421969
		//http://www.tianyancha.com/company/26079830
		.addUrl("http://www.tianyancha.com/")
		.run();
		
//		String url = null;
//		for(int i = 1; i < 100000; i++)
//		{
//			url = "http://www.tianyancha.com/company/"+i;
//			spider.addUrl(url).run();;
//		}
		
//		TianYanCha huiCong = new TianYanCha();
//		Spider spider = Spider.create(huiCong);
//		spider
//		//.setScheduler(new PriorityScheduler())
//		//.addPipeline(new DataPipeline())
//		//http://www.tianyancha.com/company/2338421969
//		//http://www.tianyancha.com/company/26079830
//		.addUrl("http://www.tianyancha.com/");
//		//.run();
//		String html = null;
//		html = JSUtil.getParseredHtml2("http://www.tianyancha.com/");
//        Page page2 = new Page();
//    	page2.setRawText(html);
//		if(page2.getHtml() == null)
//		{
//			System.out.println("null");
//		}
//	
//		System.out.println("得到最终的html5: "+page2.getHtml().toString());
		
	}

	
}